iptables -A INPUT -p tcp -m multiport --dports 6443,2379,2380,10250 -j ACCEPT

[root@master ~]#swapoff -a
[root@master ~]#
[root@master ~]# free -m
              total        used        free      shared  buff/cache   available
Mem:            976         366         135           6         474         393
Swap:             0           0           0

# swap 一栏为0，表示已经关闭了swap

# 设置加载br_netfilter模块
cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
overlay
br_netfilter
EOF

sudo modprobe overlay
sudo modprobe br_netfilter

# 开启bridge-nf-call-iptables ，设置所需的 sysctl 参数，参数在重新启动后保持不变
cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables  = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward                 = 1
EOF

# 应用 sysctl 参数而不重新启动
sudo sysctl --system

# for ubuntu
apt install -y containerd.io

containerd config default > /etc/containerd/config.toml

[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
  ...
  [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
    SystemdCgroup = true

systemctl restart containerd

# for ubuntu
apt install -y docker.io

# 修改docker进程管理器
vi /etc/docker/daemon.json
{
"exec-opts": ["native.cgroupdriver=systemd"]
}
systemctl daemon-reload && systemctl restart docker
docker info | grep -i cgroup

# 以Ubuntu系统为例

# 安装仓库依赖
sudo apt-get update
sudo apt-get install -y apt-transport-https ca-certificates curl

# use google registry
sudo curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg
echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee /etc/apt/sources.list.d/kubernetes.list

# or use aliyun registry
curl -s https://mirrors.aliyun.com/kubernetes/apt/doc/apt-key.gpg | sudo apt-key add -
tee /etc/apt/sources.list.d/kubernetes.list <<EOF 
deb https://mirrors.aliyun.com/kubernetes/apt/ kubernetes-xenial main
EOF

# 安装指定版本的kubeadm, kubelet, kubectl
apt-get update
apt-get install -y kubelet=1.24.2-00 kubeadm=1.24.2-00 kubectl=1.24.2-00

# 查询有哪些版本
apt-cache madison kubeadm

#!/bin/bash
Version=${Version:-1.24.2}
wget https://dl.k8s.io/release/v${Version}/bin/linux/amd64/kubeadm
wget https://dl.k8s.io/release/v${Version}/bin/linux/amd64/kubelet
wget https://dl.k8s.io/release/v${Version}/bin/linux/amd64/kubectl
chmod +x kubeadm kubelet kubectl
cp kubeadm kubelet kubectl /usr/bin/

# add kubelet serivce
cat > /lib/systemd/system/kubelet.service << EOF
[Unit]
Description=kubelet: The Kubernetes Node Agent
Documentation=https://kubernetes.io/docs/home/
Wants=network-online.target
After=network-online.target

[Service]
ExecStart=/usr/bin/kubelet
Restart=always
StartLimitInterval=0
RestartSec=10

[Install]
WantedBy=multi-user.target
EOF

mkdir -p /etc/systemd/system/kubelet.service.d
cat > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf << EOF
# Note: This dropin only works with kubeadm and kubelet v1.11+
[Service]
Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf"
Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml"
# This is a file that "kubeadm init" and "kubeadm join" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically
EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env
# This is a file that the user can use for overrides of the kubelet args as a last resort. Preferably, the user should use
# the .NodeRegistration.KubeletExtraArgs object in the configuration files instead. KUBELET_EXTRA_ARGS should be sourced from this file.
EnvironmentFile=-/etc/default/kubelet
ExecStart=
ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS
EOF


systemctl daemon-reload
systemctl restart kubelet

apiVersion: kubeadm.k8s.io/v1beta3
kind: InitConfiguration

apiVersion: kubeadm.k8s.io/v1beta3
kind: ClusterConfiguration

apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration

apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration

apiVersion: kubeadm.k8s.io/v1beta3
kind: JoinConfiguration

kubeadm config print init-defaults
kubeadm config print join-defaults

apiVersion: kubeadm.k8s.io/v1beta3
kind: InitConfiguration
bootstrapTokens:
  ...
nodeRegistration:
  ...

apiVersion: kubeadm.k8s.io/v1beta3
kind: ClusterConfiguration
networking:
  ...
etcd:
  ...
apiServer:
  extraArgs:
    ...
  extraVolumes:
    ...
...

kubeadm config print init-defaults > kubeadm-config.yaml

apiVersion: kubeadm.k8s.io/v1beta3
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: abcdef.0123456789abcdef
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 1.2.3.4 # 修改为apiserver的IP 或者去掉localAPIEndpoint则会读取默认IP。
  bindPort: 6443
nodeRegistration:
  criSocket: unix:///var/run/containerd/containerd.sock
  imagePullPolicy: IfNotPresent
  name: node
  taints: null
---
apiServer:
  certSANs:
  - lb.k8s.domain  # 添加额外的apiserver的域名
  - <vip/lb_ip>
  timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns: {}   # 默认为coredns
etcd:
  local:
    dataDir: /data/etcd   # 修改etcd的存储盘目录
imageRepository: k8s.gcr.io  # 修改镜像仓库地址
controlPlaneEndpoint: lb.k8s.domain  # 修改控制面域名
kind: ClusterConfiguration
kubernetesVersion: 1.24.0  # k8s 版本
networking:
  dnsDomain: cluster.local
  serviceSubnet: 10.96.0.0/12
  podSubnet: 10.244.0.0/16  # 设置pod的IP范围
scheduler: {}
---
kind: KubeletConfiguration
apiVersion: kubelet.config.k8s.io/v1beta1
cgroupDriver: systemd   # 设置为systemd

kubectl get cm -n kube-system kubeadm-config -oyaml

kubeadm config images pull

sudo kubeadm init --config kubeadm-config.yaml --upload-certs  --node-name <nodename>

...
You can now join any number of control-plane node by running the following command on each as a root:
    kubeadm join 192.168.0.200:6443 --token 9vr73a.a8uxyaju799qwdjv --discovery-token-ca-cert-hash sha256:7c2e69131a36ae2a042a339b33381c6d0d43887e2de83720eff5359e26aec866 --control-plane --certificate-key f8902e114ef118304e561c3ecd4d0b543adc226b7a07f675f56564185ffe0c07

Please note that the certificate-key gives access to cluster sensitive data, keep it secret!
As a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use kubeadm init phase upload-certs to reload certs afterward.

Then you can join any number of worker nodes by running the following on each as root:
    kubeadm join 192.168.0.200:6443 --token 9vr73a.a8uxyaju799qwdjv --discovery-token-ca-cert-hash sha256:7c2e69131a36ae2a042a339b33381c6d0d43887e2de83720eff5359e26aec866

kubeadm join <control-plane-endpoint>:6443 --token <token> \
--discovery-token-ca-cert-hash sha256:<hash> \
--control-plane --certificate-key <certificate-key> \
--node-name <nodename>

kubeadm join <control-plane-endpoint>:6443 --token <token> \
--discovery-token-ca-cert-hash sha256:<hash> \
--cri-socket /run/containerd/containerd.sock \
--node-name <nodename>

## 如果安装之后node的状态都改为ready，即为成功
wget https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
kubectl apply -f ./kube-flannel.yml
kubectl get nodes

  Warning  FailedCreatePodSandBox  4m6s                kubelet            Failed to create pod sandbox: rpc error: code = Unknown desc = failed to setup network for sandbox "300d9b570cc1e23b6335c407b8e7d0ef2c74dc2fe5d7a110678c2dc919c62edf": plugin type="flannel" failed (add): failed to delegate add: failed to set bridge addr: "cni0" already has an IP address different from 10.244.3.1/24

ifconfig cni0 down    
ip link delete cni0

kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.5.0/aio/deploy/recommended.yaml

# kubeadm重置
kubeadm reset

# 清空数据目录
rm -fr /data/etcd
rm -fr /etc/kubernetes
rm -fr ~/.kube/

ifconfig cni0 down
ip link delete cni0
ifconfig flannel.1 down
ip link delete flannel.1
rm -rf /var/lib/cni/
rm -f /etc/cni/net.d/*

[discovery] The cluster-info ConfigMap does not yet contain a JWS signature for token ID "abcdef", will try again

# 重新生成token
kubeadm token create --print-join-command
kubeadm token list

# kubeadm token create
oumnnc.aqlxuvdbntlvzoiv

# 重新生成hash
openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //'

# 查看cluster-info
kubectl -n kube-public get configmaps cluster-info -o yaml

# 修改cluster-info
kubectl -n kube-public edit configmaps cluster-info

clusters:
- cluster:
    certificate-authority-data: xxx
    server: https://lb.k8s.domain:36443
  name: ""

ansible>=2.4.0
jinja2>=2.9.6
netaddr
pbr>=1.6
ansible-modules-hashivault>=3.9.4
hvac

# 安装pip
yum -y install epel-release
yum -y install python-pip
# 安装python-netaddr
pip install netaddr

# Jinja 2.9 (or newer)
pip install --upgrade jinja2

systemctl stop firewalld
systemctl disable firewalld
iptables -F

- name: Stop if swap enabled
  assert:
    that: ansible_swaptotal_mb == 0
  when: kubelet_fail_swap_on|default(true)
  ignore_errors: "{{ ignore_assert_errors }}"

[root@master ~]#swapoff -a
[root@master ~]#
[root@master ~]# free -m
              total        used        free      shared  buff/cache   available
Mem:            976         366         135           6         474         393
Swap:             0           0           0

# swap 一栏为0，表示已经关闭了swap

- name: Stop if memory is too small for masters
  assert:
    that: ansible_memtotal_mb >= 1500
  ignore_errors: "{{ ignore_assert_errors }}"
  when: inventory_hostname in groups['kube-master']

- name: Stop if memory is too small for nodes
  assert:
    that: ansible_memtotal_mb >= 1024
  ignore_errors: "{{ ignore_assert_errors }}"
  when: inventory_hostname in groups['kube-node']

git clone https://github.com/kubernetes-incubator/kubespray.git

cd kubespray
# 复制一份配置进行修改
cp -rfp inventory/sample inventory/k8s
vi inventory/k8s/hosts.ini

# Configure 'ip' variable to bind kubernetes services on a
# different ip than the default iface
# 主机名             ssh登陆IP                        ssh用户名               ssh登陆密码                 机器IP          子网掩码
kube-master-0     ansible_ssh_host=172.16.94.140   ansible_ssh_user=root   ansible_ssh_pass=123  ip=172.16.94.140   mask=/24
kube-node-41      ansible_ssh_host=172.16.94.141   ansible_ssh_user=root   ansible_ssh_pass=123  ip=172.16.94.141   mask=/24
kube-node-42      ansible_ssh_host=172.16.94.142   ansible_ssh_user=root   ansible_ssh_pass=123  ip=172.16.94.142   mask=/24

# configure a bastion host if your nodes are not directly reachable
# bastion ansible_ssh_host=x.x.x.x

[kube-master]
kube-master-0

[etcd]
kube-master-0

[kube-node]
kube-node-41
kube-node-42

[k8s-cluster:children]
kube-node
kube-master

[calico-rr]

# 进入主目录
cd kubespray
# 执行部署命令
ansible-playbook -i inventory/k8s/hosts.ini cluster.yml -b -vvv

ansible-playbook -i inventory/k8s/hosts.ini reset.yml -b -vvv

PLAY RECAP *****************************************************************************
kube-master-0              : ok=309  changed=30   unreachable=0    failed=0
kube-node-41               : ok=203  changed=8    unreachable=0    failed=0
kube-node-42               : ok=203  changed=8    unreachable=0    failed=0
localhost                  : ok=2    changed=0    unreachable=0    failed=0

kubernetes/preinstall : Update package management cache (YUM) --------------------23.96s
/root/gopath/src/kubespray/roles/kubernetes/preinstall/tasks/main.yml:121 
kubernetes/master : Master | wait for the apiserver to be running ----------------23.44s
/root/gopath/src/kubespray/roles/kubernetes/master/handlers/main.yml:79 
kubernetes/preinstall : Install packages requirements ----------------------------20.20s
/root/gopath/src/kubespray/roles/kubernetes/preinstall/tasks/main.yml:203 
kubernetes/secrets : Check certs | check if a cert already exists on node --------13.94s
/root/gopath/src/kubespray/roles/kubernetes/secrets/tasks/check-certs.yml:17 
gather facts from all instances --------------------------------------------------9.98s
/root/gopath/src/kubespray/cluster.yml:25 
kubernetes/node : install | Compare host kubelet with hyperkube container --------9.66s
/root/gopath/src/kubespray/roles/kubernetes/node/tasks/install_host.yml:2 
kubernetes-apps/ansible : Kubernetes Apps | Start Resources -----------------------9.27s
/root/gopath/src/kubespray/roles/kubernetes-apps/ansible/tasks/main.yml:37 
kubernetes-apps/ansible : Kubernetes Apps | Lay Down KubeDNS Template ------------8.47s
/root/gopath/src/kubespray/roles/kubernetes-apps/ansible/tasks/kubedns.yml:3
download : Sync container ---------------------------------------------------------8.23s
/root/gopath/src/kubespray/roles/download/tasks/main.yml:15 
kubernetes-apps/network_plugin/calico : Start Calico resources --------------------7.82s
/root/gopath/src/kubespray/roles/kubernetes-apps/network_plugin/calico/tasks/main.yml:2 
download : Download items ---------------------------------------------------------7.67s
/root/gopath/src/kubespray/roles/download/tasks/main.yml:6 
download : Download items ---------------------------------------------------------7.48s
/root/gopath/src/kubespray/roles/download/tasks/main.yml:6 
download : Sync container ---------------------------------------------------------7.35s
/root/gopath/src/kubespray/roles/download/tasks/main.yml:15 
download : Download items ---------------------------------------------------------7.16s
/root/gopath/src/kubespray/roles/download/tasks/main.yml:6 
network_plugin/calico : Calico | Copy cni plugins from calico/cni container -------7.10s
/root/gopath/src/kubespray/roles/network_plugin/calico/tasks/main.yml:62 
download : Download items ---------------------------------------------------------7.04s
/root/gopath/src/kubespray/roles/download/tasks/main.yml:6
download : Download items ---------------------------------------------------------7.01s
/root/gopath/src/kubespray/roles/download/tasks/main.yml:6 
download : Sync container ---------------------------------------------------------7.00s
/root/gopath/src/kubespray/roles/download/tasks/main.yml:15 
download : Download items ---------------------------------------------------------6.98s
/root/gopath/src/kubespray/roles/download/tasks/main.yml:6 
download : Download items ---------------------------------------------------------6.79s
/root/gopath/src/kubespray/roles/download/tasks/main.yml:6 

# kubectl get all --namespace=kube-system
NAME             DESIRED   CURRENT   READY     UP-TO-DATE   AVAILABLE   NODE SELECTOR   AGE
ds/calico-node   3         3         3         3            3           <none>          2h

NAME                          DESIRED   CURRENT   UP-TO-DATE   AVAILABLE   AGE
deploy/kube-dns               2         2         2            2           2h
deploy/kubedns-autoscaler     1         1         1            1           2h
deploy/kubernetes-dashboard   1         1         1            1           2h

NAME                                 DESIRED   CURRENT   READY     AGE
rs/kube-dns-79d99cdcd5               2         2         2         2h
rs/kubedns-autoscaler-5564b5585f     1         1         1         2h
rs/kubernetes-dashboard-69cb58d748   1         1         1         2h

NAME             DESIRED   CURRENT   READY     UP-TO-DATE   AVAILABLE   NODE SELECTOR   AGE
ds/calico-node   3         3         3         3            3           <none>          2h

NAME                          DESIRED   CURRENT   UP-TO-DATE   AVAILABLE   AGE
deploy/kube-dns               2         2         2            2           2h
deploy/kubedns-autoscaler     1         1         1            1           2h
deploy/kubernetes-dashboard   1         1         1            1           2h

NAME                                 DESIRED   CURRENT   READY     AGE
rs/kube-dns-79d99cdcd5               2         2         2         2h
rs/kubedns-autoscaler-5564b5585f     1         1         1         2h
rs/kubernetes-dashboard-69cb58d748   1         1         1         2h

NAME                                       READY     STATUS    RESTARTS   AGE
po/calico-node-22vsg                       1/1       Running   0          2h
po/calico-node-t7zgw                       1/1       Running   0          2h
po/calico-node-zqnx8                       1/1       Running   0          2h
po/kube-apiserver-kube-master-0            1/1       Running   0          22h
po/kube-controller-manager-kube-master-0   1/1       Running   0          2h
po/kube-dns-79d99cdcd5-f2t6t               3/3       Running   0          2h
po/kube-dns-79d99cdcd5-gw944               3/3       Running   0          2h
po/kube-proxy-kube-master-0                1/1       Running   2          22h
po/kube-proxy-kube-node-41                 1/1       Running   3          22h
po/kube-proxy-kube-node-42                 1/1       Running   3          22h
po/kube-scheduler-kube-master-0            1/1       Running   0          2h
po/kubedns-autoscaler-5564b5585f-lt9bb     1/1       Running   0          2h
po/kubernetes-dashboard-69cb58d748-wmb9x   1/1       Running   0          2h
po/nginx-proxy-kube-node-41                1/1       Running   3          22h
po/nginx-proxy-kube-node-42                1/1       Running   3          22h

NAME                       TYPE        CLUSTER-IP     EXTERNAL-IP   PORT(S)         AGE
svc/kube-dns               ClusterIP   10.233.0.3     <none>        53/UDP,53/TCP   2h
svc/kubernetes-dashboard   ClusterIP   10.233.27.24   <none>        443/TCP         2h

# kubectl get nodes
NAME            STATUS    ROLES     AGE       VERSION
kube-master-0   Ready     master    22h       v1.9.5
kube-node-41    Ready     node      22h       v1.9.5
kube-node-42    Ready     node      22h       v1.9.5

# kubectl get cs
NAME                 STATUS    MESSAGE              ERROR
scheduler            Healthy   ok
controller-manager   Healthy   ok
etcd-0               Healthy   {"health": "true"}

# Configure 'ip' variable to bind kubernetes services on a
# different ip than the default iface
# 主机名             ssh登陆IP                        ssh用户名               ssh登陆密码                 机器IP          子网掩码
kube-master-0     ansible_ssh_host=172.16.94.140   ansible_ssh_user=root   ansible_ssh_pass=123  ip=172.16.94.140   mask=/24
kube-node-41      ansible_ssh_host=172.16.94.141   ansible_ssh_user=root   ansible_ssh_pass=123  ip=172.16.94.141   mask=/24
kube-node-42      ansible_ssh_host=172.16.94.142   ansible_ssh_user=root   ansible_ssh_pass=123  ip=172.16.94.142   mask=/24
kube-node-43      ansible_ssh_host=172.16.94.143   ansible_ssh_user=root   ansible_ssh_pass=123  ip=172.16.94.143   mask=/24

# configure a bastion host if your nodes are not directly reachable
# bastion ansible_ssh_host=x.x.x.x

[kube-master]
kube-master-0

[etcd]
kube-master-0

[kube-node]
kube-node-41
kube-node-42
kube-node-43

[k8s-cluster:children]
kube-node
kube-master

[calico-rr]

# 进入主目录
cd kubespray
# 执行部署命令
ansible-playbook -i inventory/k8s/hosts.ini scale.yml -b -vvv

PLAY RECAP ***************************************
kube-node-41               : ok=228  changed=11   unreachable=0    failed=0
kube-node-42               : ok=197  changed=6    unreachable=0    failed=0
kube-node-43               : ok=227  changed=69   unreachable=0    failed=0 # 新增Node节点
localhost                  : ok=2    changed=0    unreachable=0    failed=0

# kubectl get nodes
NAME            STATUS    ROLES     AGE       VERSION
kube-master-0   Ready     master    1d        v1.9.5
kube-node-41    Ready     node      1d        v1.9.5
kube-node-42    Ready     node      1d        v1.9.5
kube-node-43    Ready     node      1m        v1.9.5   #该节点为新增Node节点

# kubectl get po --namespace=kube-system -o wide
NAME                                    READY     STATUS    RESTARTS   AGE       IP               NODE
calico-node-22vsg                       1/1       Running   0          10h       172.16.94.140    kube-master-0
calico-node-8fz9x                       1/1       Running   2          27m       172.16.94.143    kube-node-43
calico-node-t7zgw                       1/1       Running   0          10h       172.16.94.142    kube-node-42
calico-node-zqnx8                       1/1       Running   0          10h       172.16.94.141    kube-node-41
kube-apiserver-kube-master-0            1/1       Running   0          1d        172.16.94.140    kube-master-0
kube-controller-manager-kube-master-0   1/1       Running   0          10h       172.16.94.140    kube-master-0
kube-dns-79d99cdcd5-f2t6t               3/3       Running   0          10h       10.233.100.194   kube-node-41
kube-dns-79d99cdcd5-gw944               3/3       Running   0          10h       10.233.107.1     kube-node-42
kube-proxy-kube-master-0                1/1       Running   2          1d        172.16.94.140    kube-master-0
kube-proxy-kube-node-41                 1/1       Running   3          1d        172.16.94.141    kube-node-41
kube-proxy-kube-node-42                 1/1       Running   3          1d        172.16.94.142    kube-node-42
kube-proxy-kube-node-43                 1/1       Running   0          26m       172.16.94.143    kube-node-43
kube-scheduler-kube-master-0            1/1       Running   0          10h       172.16.94.140    kube-master-0
kubedns-autoscaler-5564b5585f-lt9bb     1/1       Running   0          10h       10.233.100.193   kube-node-41
kubernetes-dashboard-69cb58d748-wmb9x   1/1       Running   0          10h       10.233.107.2     kube-node-42
nginx-proxy-kube-node-41                1/1       Running   3          1d        172.16.94.141    kube-node-41
nginx-proxy-kube-node-42                1/1       Running   3          1d        172.16.94.142    kube-node-42
nginx-proxy-kube-node-43                1/1       Running   0          26m       172.16.94.143    kube-node-43

ansible-playbook -i inventory/k8s/hosts.ini cluster.yml -b -vvv

# Configure 'ip' variable to bind kubernetes services on a
# different ip than the default iface
# 主机名             ssh登陆IP                        ssh用户名               ssh登陆密码                 机器IP          子网掩码
kube-master-0     ansible_ssh_host=172.16.94.140   ansible_ssh_user=root   ansible_ssh_pass=123  ip=172.16.94.140   mask=/24
kube-master-1     ansible_ssh_host=172.16.94.144   ansible_ssh_user=root   ansible_ssh_pass=123  ip=172.16.94.144   mask=/24
kube-master-2     ansible_ssh_host=172.16.94.145   ansible_ssh_user=root   ansible_ssh_pass=123  ip=172.16.94.145   mask=/24
kube-node-41      ansible_ssh_host=172.16.94.141   ansible_ssh_user=root   ansible_ssh_pass=123  ip=172.16.94.141   mask=/24
kube-node-42      ansible_ssh_host=172.16.94.142   ansible_ssh_user=root   ansible_ssh_pass=123  ip=172.16.94.142   mask=/24
kube-node-43      ansible_ssh_host=172.16.94.143   ansible_ssh_user=root   ansible_ssh_pass=123  ip=172.16.94.143   mask=/24

# configure a bastion host if your nodes are not directly reachable
# bastion ansible_ssh_host=x.x.x.x

[kube-master]
kube-master-0
kube-master-1
kube-master-2

[etcd]
kube-master-0
kube-master-1
kube-master-2

[kube-node]
kube-node-41
kube-node-42
kube-node-43

[k8s-cluster:children]
kube-node
kube-master

[calico-rr]

ansible-playbook upgrade-cluster.yml -b -i inventory/k8s/hosts.ini -e kube_version=v1.10.4 -vvv

fatal: [node1]: FAILED! => {"failed": true, "msg": "The ipaddr filter requires python-netaddr be installed on the ansible controller"}

fatal: [kube-master-0]: FAILED! => {
    "assertion": "ansible_swaptotal_mb == 0",
    "changed": false,
    "evaluated_to": false
}
fatal: [kube-node-41]: FAILED! => {
    "assertion": "ansible_swaptotal_mb == 0",
    "changed": false,
    "evaluated_to": false
}
fatal: [kube-node-42]: FAILED! => {
    "assertion": "ansible_swaptotal_mb == 0",
    "changed": false,
    "evaluated_to": false
}

TASK [kubernetes/preinstall : Stop if memory is too small for masters] *********************************************************************************************************************************************************************************************************
task path: /root/gopath/src/kubespray/roles/kubernetes/preinstall/tasks/verify-settings.yml:52
Friday 10 August 2018  21:50:26 +0800 (0:00:00.940)       0:01:14.088 *********
fatal: [kube-master-0]: FAILED! => {
    "assertion": "ansible_memtotal_mb >= 1500",
    "changed": false,
    "evaluated_to": false
}

TASK [kubernetes/preinstall : Stop if memory is too small for nodes] ***********************************************************************************************************************************************************************************************************
task path: /root/gopath/src/kubespray/roles/kubernetes/preinstall/tasks/verify-settings.yml:58
Friday 10 August 2018  21:50:27 +0800 (0:00:00.570)       0:01:14.659 *********
fatal: [kube-node-41]: FAILED! => {
    "assertion": "ansible_memtotal_mb >= 1024",
    "changed": false,
    "evaluated_to": false
}
fatal: [kube-node-42]: FAILED! => {
    "assertion": "ansible_memtotal_mb >= 1024",
    "changed": false,
    "evaluated_to": false
}
	to retry, use: --limit @/root/gopath/src/kubespray/cluster.retry

FAILED - RETRYING: Master | wait for kube-scheduler (1 retries left).
FAILED - RETRYING: Master | wait for kube-scheduler (1 retries left).
fatal: [node1]: FAILED! => {"attempts": 60, "changed": false, "content": "", "failed": true, "msg": "Status code was not [200]: Request failed: <urlopen error [Errno 111] Connection refused>", "redirected": false, "status": -1, "url": "http://localhost:10251/healthz"}

failed: [k8s-node-1] (item={u'name': u'docker-engine-1.13.1-1.el7.centos'}) => {
    "attempts": 4,
    "changed": false,
    ...
    "item": {
        "name": "docker-engine-1.13.1-1.el7.centos"
    },
    "msg": "Error: docker-ce-selinux conflicts with 2:container-selinux-2.66-1.el7.noarch\n",
    "rc": 1,
    "results": [
        "Loaded plugins: fastestmirror\nLoading mirror speeds from cached hostfile\n * elrepo: mirrors.tuna.tsinghua.edu.cn\n * epel: mirrors.tongji.edu.cn\nPackage docker-engine is obsoleted by docker-ce, trying to install docker-ce-17.03.2.ce-1.el7.centos.x86_64 instead\nResolving Dependencies\n--> Running transaction check\n---> Package docker-ce.x86_64 0:17.03.2.ce-1.el7.centos will be installed\n--> Processing Dependency: docker-ce-selinux >= 17.03.2.ce-1.el7.centos for package: docker-ce-17.03.2.ce-1.el7.centos.x86_64\n--> Processing Dependency: libltdl.so.7()(64bit) for package: docker-ce-17.03.2.ce-1.el7.centos.x86_64\n--> Running transaction check\n---> Package docker-ce-selinux.noarch 0:17.03.2.ce-1.el7.centos will be installed\n---> Package libtool-ltdl.x86_64 0:2.4.2-22.el7_3 will be installed\n--> Processing Conflict: docker-ce-selinux-17.03.2.ce-1.el7.centos.noarch conflicts docker-selinux\n--> Restarting Dependency Resolution with new changes.\n--> Running transaction check\n---> Package container-selinux.noarch 2:2.55-1.el7 will be updated\n---> Package container-selinux.noarch 2:2.66-1.el7 will be an update\n--> Processing Conflict: docker-ce-selinux-17.03.2.ce-1.el7.centos.noarch conflicts docker-selinux\n--> Finished Dependency Resolution\n You could try using --skip-broken to work around the problem\n You could try running: rpm -Va --nofiles --nodigest\n"
    ]
}

sudo yum remove -y docker \
                  docker-client \
                  docker-client-latest \
                  docker-common \
                  docker-latest \
                  docker-latest-logrotate \
                  docker-logrotate \
                  docker-selinux \
                  docker-engine-selinux \
                  docker-engine

curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin/

curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.9.0/bin/linux/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin/

curl -Lo minikube https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 && chmod +x minikube && sudo mv minikube /usr/local/bin/

curl -Lo minikube https://storage.googleapis.com/minikube/releases/v0.28.2/minikube-linux-amd64 && chmod +x minikube && sudo mv minikube /usr/local/bin/

Minikube is a CLI tool that provisions and manages single-node Kubernetes clusters optimized for development workflows.

Usage:
  minikube [command]

Available Commands:
  addons           Modify minikube's kubernetes addons
  cache            Add or delete an image from the local cache.
  completion       Outputs minikube shell completion for the given shell (bash or zsh)
  config           Modify minikube config
  dashboard        Opens/displays the kubernetes dashboard URL for your local cluster
  delete           Deletes a local kubernetes cluster
  docker-env       Sets up docker env variables; similar to '$(docker-machine env)'
  get-k8s-versions Gets the list of Kubernetes versions available for minikube when using the localkube bootstrapper
  ip               Retrieves the IP address of the running cluster
  logs             Gets the logs of the running localkube instance, used for debugging minikube, not user code
  mount            Mounts the specified directory into minikube
  profile          Profile sets the current minikube profile
  service          Gets the kubernetes URL(s) for the specified service in your local cluster
  ssh              Log into or run a command on a machine with SSH; similar to 'docker-machine ssh'
  ssh-key          Retrieve the ssh identity key path of the specified cluster
  start            Starts a local kubernetes cluster
  status           Gets the status of a local kubernetes cluster
  stop             Stops a running local kubernetes cluster
  update-check     Print current and latest version number
  update-context   Verify the IP address of the running cluster in kubeconfig.
  version          Print the version of minikube

Flags:
      --alsologtostderr                  log to standard error as well as files
  -b, --bootstrapper string              The name of the cluster bootstrapper that will set up the kubernetes cluster. (default "localkube")
      --log_backtrace_at traceLocation   when logging hits line file:N, emit a stack trace (default :0)
      --log_dir string                   If non-empty, write log files in this directory
      --loglevel int                     Log level (0 = DEBUG, 5 = FATAL) (default 1)
      --logtostderr                      log to standard error instead of files
  -p, --profile string                   The name of the minikube VM being used.
	This can be modified to allow for multiple minikube instances to be run independently (default "minikube")
      --stderrthreshold severity         logs at or above this threshold go to stderr (default 2)
  -v, --v Level                          log level for V logs
      --vmodule moduleSpec               comma-separated list of pattern=N settings for file-filtered logging

Use "minikube [command] --help" for more information about a command.

minikube start --vm-driver=none

root@ubuntu:~# minikube start --vm-driver=none
Starting local Kubernetes v1.10.0 cluster...
Starting VM...
Getting VM IP address...
Moving files into cluster...
Downloading kubeadm v1.10.0
Downloading kubelet v1.10.0
^[[DFinished Downloading kubelet v1.10.0
Finished Downloading kubeadm v1.10.0
Setting up certs...
Connecting to cluster...
Setting up kubeconfig...
Starting cluster components...
Kubectl is now configured to use the cluster.
===================
WARNING: IT IS RECOMMENDED NOT TO RUN THE NONE DRIVER ON PERSONAL WORKSTATIONS
	The 'none' driver will run an insecure kubernetes apiserver as root that may leave the host vulnerable to CSRF attacks

When using the none driver, the kubectl config and credentials generated will be root owned and will appear in the root home directory.
You will need to move the files to the appropriate location and then set the correct permissions.  An example of this is below:

	sudo mv /root/.kube $HOME/.kube # this will write over any previous configuration
	sudo chown -R $USER $HOME/.kube
	sudo chgrp -R $USER $HOME/.kube

	sudo mv /root/.minikube $HOME/.minikube # this will write over any previous configuration
	sudo chown -R $USER $HOME/.minikube
	sudo chgrp -R $USER $HOME/.minikube

This can also be done automatically by setting the env var CHANGE_MINIKUBE_NONE_USER=true
Loading cached images from config file.

# 查阅版本
minikube get-k8s-versions
# 选择版本启动
minikube start --kubernetes-version v1.7.3 --vm-driver=none

$ minikube status
minikube: Running
cluster: Running
kubectl: Correctly Configured: pointing to minikube-vm at 172.16.94.139

root@ubuntu:~# kubectl get all --namespace=kube-system
NAME                                        READY     STATUS    RESTARTS   AGE
pod/etcd-minikube                           1/1       Running   0          38m
pod/kube-addon-manager-minikube             1/1       Running   0          38m
pod/kube-apiserver-minikube                 1/1       Running   1          39m
pod/kube-controller-manager-minikube        1/1       Running   0          38m
pod/kube-dns-86f4d74b45-bdfnx               3/3       Running   0          38m
pod/kube-proxy-dqdvg                        1/1       Running   0          38m
pod/kube-scheduler-minikube                 1/1       Running   0          38m
pod/kubernetes-dashboard-5498ccf677-c2gnh   1/1       Running   0          38m
pod/storage-provisioner                     1/1       Running   0          38m

NAME                           TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)         AGE
service/kube-dns               ClusterIP   10.96.0.10      <none>        53/UDP,53/TCP   38m
service/kubernetes-dashboard   NodePort    10.104.48.227   <none>        80:30000/TCP    38m

NAME                        DESIRED   CURRENT   READY     UP-TO-DATE   AVAILABLE   NODE SELECTOR   AGE
daemonset.apps/kube-proxy   1         1         1         1            1           <none>          38m

NAME                                   DESIRED   CURRENT   UP-TO-DATE   AVAILABLE   AGE
deployment.apps/kube-dns               1         1         1            1           38m
deployment.apps/kubernetes-dashboard   1         1         1            1           38m

NAME                                              DESIRED   CURRENT   READY     AGE
replicaset.apps/kube-dns-86f4d74b45               1         1         1         38m
replicaset.apps/kubernetes-dashboard-5498ccf677   1         1         1         38m

[root@minikube ~]# minikube start
Starting local Kubernetes v1.10.0 cluster...
Starting VM...
Downloading Minikube ISO
 160.27 MB / 160.27 MB [============================================] 100.00% 0s
E0727 15:47:08.655647    9407 start.go:174] Error starting host: Error creating host: Error executing step: Running precreate checks.
: VBoxManage not found. Make sure VirtualBox is installed and VBoxManage is in the path.

 Retrying.
E0727 15:47:08.656994    9407 start.go:180] Error starting host:  Error creating host: Error executing step: Running precreate checks.
: VBoxManage not found. Make sure VirtualBox is installed and VBoxManage is in the path
================================================================================
An error has occurred. Would you like to opt in to sending anonymized crash
information to minikube to help prevent future errors?
To opt out of these messages, run the command:
	minikube config set WantReportErrorPrompt false
================================================================================
Please enter your response [Y/n]:

[root@minikube ~]# minikube start --vm-driver=none
Starting local Kubernetes v1.10.0 cluster...
Starting VM...
E0727 15:56:54.936706    9441 start.go:174] Error starting host: Error creating host: Error executing step: Running precreate checks.
: docker cannot be found on the path for this machine. A docker installation is a requirement for using the none driver: exec: "docker": executable file not found in $PATH.

 Retrying.
E0727 15:56:54.938930    9441 start.go:180] Error starting host:  Error creating host: Error executing step: Running precreate checks.
: docker cannot be found on the path for this machine. A docker installation is a requirement for using the none driver: exec: "docker": executable file not found in $PATH

curl -Lo ./kind "https://github.com/kubernetes-sigs/kind/releases/download/v0.7.0/kind-$(uname)-amd64"
chmod +x ./kind
mv ./kind /some-dir-in-your-PATH/kind

$ kind create cluster
Creating cluster "kind" ...
 ✓ Ensuring node image (kindest/node:v1.17.0) 🖼
 ✓ Preparing nodes 📦
 ✓ Writing configuration 📜
 ✓ Starting control-plane 🕹️
 ✓ Installing CNI 🔌
 ✓ Installing StorageClass 💾
Set kubectl context to "kind-kind"
You can now use your cluster with:

kubectl cluster-info --context kind-kind

Not sure what to do next? 😅 Check out https://kind.sigs.k8s.io/docs/user/quick-start/

$ kubectl cluster-info --context kind-kind
Kubernetes master is running at https://127.0.0.1:32768
KubeDNS is running at https://127.0.0.1:32768/api/v1/namespaces/kube-system/services/kube-dns:dns/proxy

$ kubectl get node -o wide
NAME                 STATUS   ROLES    AGE   VERSION   INTERNAL-IP   EXTERNAL-IP   OS-IMAGE       KERNEL-VERSION                      CONTAINER-RUNTIME
kind-control-plane   Ready    master   35h   v1.17.0   172.17.0.2    <none>        Ubuntu 19.10   3.10.107-1-tlinux2_kvm_guest-0049   containerd://1.3.2

$ kubectl get po --all-namespaces -o wide
NAMESPACE            NAME                                         READY   STATUS    RESTARTS   AGE   IP           NODE                 NOMINATED NODE   READINESS GATES
kube-system          coredns-6955765f44-lqk9v                     1/1     Running   0          35h   10.244.0.4   kind-control-plane   <none>           <none>
kube-system          coredns-6955765f44-zpsmc                     1/1     Running   0          35h   10.244.0.3   kind-control-plane   <none>           <none>
kube-system          etcd-kind-control-plane                      1/1     Running   0          35h   172.17.0.2   kind-control-plane   <none>           <none>
kube-system          kindnet-8mt7d                                1/1     Running   0          35h   172.17.0.2   kind-control-plane   <none>           <none>
kube-system          kube-apiserver-kind-control-plane            1/1     Running   0          35h   172.17.0.2   kind-control-plane   <none>           <none>
kube-system          kube-controller-manager-kind-control-plane   1/1     Running   0          35h   172.17.0.2   kind-control-plane   <none>           <none>
kube-system          kube-proxy-5w25s                             1/1     Running   0          35h   172.17.0.2   kind-control-plane   <none>           <none>
kube-system          kube-scheduler-kind-control-plane            1/1     Running   0          35h   172.17.0.2   kind-control-plane   <none>           <none>
local-path-storage   local-path-provisioner-7745554f7f-dckzr      1/1     Running   0          35h   10.244.0.2   kind-control-plane   <none>           <none>

$ docker ps
CONTAINER ID        IMAGE                  COMMAND                  CREATED             STATUS              PORTS                       NAMES
93b291f99dd4        kindest/node:v1.17.0   "/usr/local/bin/entr…"   2 minutes ago       Up 2 minutes        127.0.0.1:32768->6443/tcp   kind-control-plane

$ docker exec -it 93b291f99dd4 bash
root@kind-control-plane:/# ps auxw
USER       PID %CPU %MEM    VSZ   RSS TTY      STAT START   TIME COMMAND
root         1  0.1  0.0  19512  7480 ?        Ss   03:18   0:00 /sbin/init
root       105  0.0  0.0  26396  7344 ?        S<s  03:18   0:00 /lib/systemd/systemd-journald
root       141  2.3  0.3 2374736 51564 ?       Ssl  03:18   0:06 /usr/local/bin/containerd
root       325  0.0  0.0 112540  5036 ?        Sl   03:18   0:00 /usr/local/bin/containerd-shim-runc-v2 -namespace k8s.io -id 3f415d609e15ef12b9f53557891c311c156b912d5a326544a25c8b29cfa9d366 -address /run/containerd/containerd.sock
root       346  0.0  0.0   1012     4 ?        Ss   03:18   0:00 /pause
root       370  0.0  0.0 112540  5108 ?        Sl   03:18   0:00 /usr/local/bin/containerd-shim-runc-v2 -namespace k8s.io -id 1e1f3eed09f701fb621325e7b9e96d1c3de60ebd3bd64e0aec376e9490cf0e57 -address /run/containerd/containerd.sock
root       397  0.0  0.0 112540  4684 ?        Sl   03:18   0:00 /usr/local/bin/containerd-shim-runc-v2 -namespace k8s.io -id c5e451089a1a5b3dfb2cc68ee27ac7d414285be55ecfdc5bd59180fbfbc7df2e -address /run/containerd/containerd.sock
root       424  0.0  0.0 112540  4924 ?        Sl   03:18   0:00 /usr/local/bin/containerd-shim-runc-v2 -namespace k8s.io -id 81e35f29ac8c2dda344125a10e3791be7ccf788a88f1efbc3397fa319f02881f -address /run/containerd/containerd.sock
root       443  0.0  0.0   1012     4 ?        Ss   03:18   0:00 /pause
root       458  0.0  0.0   1012     4 ?        Ss   03:18   0:00 /pause
root       465  0.0  0.0   1012     4 ?        Ss   03:18   0:00 /pause
root       548  0.7  0.1 145500 27724 ?        Ssl  03:18   0:02 kube-scheduler --authentication-kubeconfig=/etc/kubernetes/scheduler.conf --authorization-kubeconfig=/etc/kubernetes/scheduler.conf --bind-address=127.0.0.1 --kubeconfig=/etc/kubernetes/scheduler.conf --leader-elect=true
root       589  1.0  0.3 159536 54384 ?        Ssl  03:18   0:02 kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/etc/kubernetes/controller-manager.conf --authorization-kubeconfig=/etc/kubernetes/controller-manager.conf --bind-address=127.0.0.1 --client-ca-file=/etc/kubernetes/pki/ca.cr
root       613  3.8  1.6 445780 273484 ?       Ssl  03:18   0:10 kube-apiserver --advertise-address=172.17.0.2 --allow-privileged=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/pki/ca.crt --enable-admission-plugins=NodeRestriction --enable-bootstrap-token-auth=true --etcd-cafile=/etc/kubernetes/
root       660  1.4  0.2 10613604 37448 ?      Ssl  03:18   0:04 etcd --advertise-client-urls=https://172.17.0.2:2379 --cert-file=/etc/kubernetes/pki/etcd/server.crt --client-cert-auth=true --data-dir=/var/lib/etcd --initial-advertise-peer-urls=https://172.17.0.2:2380 --initial-cluster=kind-control-plane=https://172.
root       718  1.3  0.3 2084848 52772 ?       Ssl  03:18   0:03 /usr/bin/kubelet --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf --config=/var/lib/kubelet/config.yaml --container-runtime=remote --container-runtime-endpoint=/run/containerd/containerd.sock --fail
root       876  0.0  0.0 112540  5084 ?        Sl   03:18   0:00 /usr/local/bin/containerd-shim-runc-v2 -namespace k8s.io -id adfbea8fec5ac6986407291f5bfc5aecead176954e5dabbe1517b98dd77bf78b -address /run/containerd/containerd.sock
root       893  0.0  0.0 112540  4796 ?        Sl   03:18   0:00 /usr/local/bin/containerd-shim-runc-v2 -namespace k8s.io -id 53bdce023626b60ffaa0548b5888e457dc9c3bc45c7808a385dd0f63dcc90327 -address /run/containerd/containerd.sock
root       924  0.0  0.0   1012     4 ?        Ss   03:18   0:00 /pause
root       931  0.0  0.0   1012     4 ?        Ss   03:18   0:00 /pause
root      1000  0.0  0.0 127616 11100 ?        Ssl  03:18   0:00 /bin/kindnetd
root      1017  0.0  0.1 141060 19420 ?        Ssl  03:18   0:00 /usr/local/bin/kube-proxy --config=/var/lib/kube-proxy/config.conf --hostname-override=kind-control-plane
root      1066  0.0  0.0      0     0 ?        Z    03:18   0:00 [iptables-nft-sa] <defunct>
root      1080  0.0  0.0      0     0 ?        Z    03:18   0:00 [iptables-nft-sa] <defunct>
root      1241  0.0  0.0 112540  5156 ?        Sl   03:19   0:00 /usr/local/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5cbd7bbe186cf5847786c7a03aa4c6f82e6c805d0a189f0f3e8fb1750594260d -address /run/containerd/containerd.sock
root      1262  0.0  0.0   1012     4 ?        Ss   03:19   0:00 /pause
root      1303  0.1  0.0 134372 14088 ?        Ssl  03:19   0:00 local-path-provisioner --debug start --helper-image k8s.gcr.io/debian-base:v2.0.0 --config /etc/config/config.json
root      1411  0.0  0.0 112540  4876 ?        Sl   03:19   0:00 /usr/local/bin/containerd-shim-runc-v2 -namespace k8s.io -id 196b440345cb5ef47a6c31222323d35bbfef85d1d79c149ec0e3a6e22022a5f0 -address /run/containerd/containerd.sock
root      1437  0.0  0.0   1012     4 ?        Ss   03:19   0:00 /pause
root      1450  0.0  0.0 112540  4380 ?        Sl   03:19   0:00 /usr/local/bin/containerd-shim-runc-v2 -namespace k8s.io -id de7bdf052083978c78708383f842567d4fb38adff22a56792437a4de82425afe -address /run/containerd/containerd.sock
root      1480  0.0  0.0   1012     4 ?        Ss   03:19   0:00 /pause
root      1530  0.1  0.1 144324 19056 ?        Ssl  03:19   0:00 /coredns -conf /etc/coredns/Corefile
root      1531  0.1  0.1 144580 19204 ?        Ssl  03:19   0:00 /coredns -conf /etc/coredns/Corefile

# 下载cfssl
$ curl https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 -o /usr/local/bin/cfssl
$ curl https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 -o /usr/local/bin/cfssljson
$ curl https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64 -o /usr/local/bin/cfssl-certinfo

# 添加可执行权限
$ chmod +x /usr/local/bin/cfssl /usr/local/bin/cfssljson /usr/local/bin/cfssl-certinfo

cat << EOF > ca-config.json
{
  "signing": {
    "default": {
      "expiry": "87600h"
    },
    "profiles": {
      "kubernetes": {
        "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ],
        "expiry": "876000h"
      }
    }
  }
}
EOF

cat << EOF > ca-csr.json
{
  "CN": "kubernetes",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "ShenZhen",
      "L": "ShenZhen",
      "O": "k8s",
      "OU": "System"
    }
  ]
}
EOF

cfssl gencert -initca ca-csr.json | cfssljson -bare ca

# cfssl gencert -initca ca-csr.json | cfssljson -bare ca
2019/12/13 14:35:52 [INFO] generating a new CA key and certificate from CSR
2019/12/13 14:35:52 [INFO] generate received request
2019/12/13 14:35:52 [INFO] received CSR
2019/12/13 14:35:52 [INFO] generating key: rsa-2048
2019/12/13 14:35:52 [INFO] encoded CSR
2019/12/13 14:35:52 [INFO] signed certificate with serial number 248379771349454958117219047414671162179070747780

# 生成文件
-rw-r--r-- 1 root root 1005 12月 13 11:32 ca.csr
-rw------- 1 root root 1675 12月 13 11:32 ca-key.pem
-rw-r--r-- 1 root root 1363 12月 13 11:32 ca.pem
# 配置源文件
-rw-r--r-- 1 root root  293 12月 13 11:31 ca-config.json
-rw-r--r-- 1 root root  210 12月 13 11:31 ca-csr.json

cat << EOF > kubernetes-csr.json
{
  "CN": "kubernetes",
  "hosts": [
    "127.0.0.1",
    "<MASTER_IP>",
    "<MASTER_CLUSTER_IP>", 
    "kubernetes",
    "kubernetes.default",
    "kubernetes.default.svc",
    "kubernetes.default.svc.cluster",
    "kubernetes.default.svc.cluster.local"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [{
    "C": "<country>",
    "ST": "<state>",
    "L": "<city>",
    "O": "<organization>",
    "OU": "<organization unit>"
  }]
}
EOF

$ cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kubernetes-csr.json | cfssljson -bare kubernetes

# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kubernetes-csr.json | cfssljson -bare kubernetes
2019/12/13 14:40:28 [INFO] generate received request
2019/12/13 14:40:28 [INFO] received CSR
2019/12/13 14:40:28 [INFO] generating key: rsa-2048
2019/12/13 14:40:28 [INFO] encoded CSR
2019/12/13 14:40:28 [INFO] signed certificate with serial number 392795299385191732458211386861696542628305189374
2019/12/13 14:40:28 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").

# 生成文件
-rw-r--r-- 1 root root 1269 12月 13 14:40 kubernetes.csr
-rw------- 1 root root 1679 12月 13 14:40 kubernetes-key.pem
-rw-r--r-- 1 root root 1643 12月 13 14:40 kubernetes.pem
# 配置源文件
-rw-r--r-- 1 root root  580 12月 13 14:40 kubernetes-csr.json

cat << EOF > admin-csr.json
{
  "CN": "admin",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "ShenZhen",
      "L": "ShenZhen",
      "O": "system:masters",
      "OU": "System"
    }
  ]
}
EOF

$ cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes admin-csr.json | cfssljson -bare admin

# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes admin-csr.json | cfssljson -bare admin
2019/12/13 14:52:37 [INFO] generate received request
2019/12/13 14:52:37 [INFO] received CSR
2019/12/13 14:52:37 [INFO] generating key: rsa-2048
2019/12/13 14:52:37 [INFO] encoded CSR
2019/12/13 14:52:37 [INFO] signed certificate with serial number 465422983473444224050765004141217688748259757371
2019/12/13 14:52:37 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").

# 生成文件
-rw-r--r-- 1 root root 1013 12月 13 14:52 admin.csr
-rw------- 1 root root 1675 12月 13 14:52 admin-key.pem
-rw-r--r-- 1 root root 1407 12月 13 14:52 admin.pem
# 配置源文件
-rw-r--r-- 1 root root  231 12月 13 14:49 admin-csr.json

cat << EOF > kube-proxy-csr.json
{
  "CN": "system:kube-proxy",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "BeiJing",
      "L": "BeiJing",
      "O": "k8s",
      "OU": "System"
    }
  ]
}
EOF

$ cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes  kube-proxy-csr.json | cfssljson -bare kube-proxy

# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes  kube-proxy-csr.json | cfssljson -bare kube-proxy
2019/12/13 19:37:48 [INFO] generate received request
2019/12/13 19:37:48 [INFO] received CSR
2019/12/13 19:37:48 [INFO] generating key: rsa-2048
2019/12/13 19:37:48 [INFO] encoded CSR
2019/12/13 19:37:48 [INFO] signed certificate with serial number 526712749765692443642491255093816136154324531741
2019/12/13 19:37:48 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").

# 生成文件
-rw-r--r-- 1 root root 1009 12月 13 19:37 kube-proxy.csr
-rw------- 1 root root 1675 12月 13 19:37 kube-proxy-key.pem
-rw-r--r-- 1 root root 1407 12月 13 19:37 kube-proxy.pem
# 配置源文件
-rw-r--r-- 1 root root  230 12月 13 19:37 kube-proxy-csr.json

openssl x509  -noout -text -in  kubernetes.pem

# openssl x509  -noout -text -in  kubernetes.pem
Certificate:
    Data:
        Version: 3 (0x2)
        Serial Number:
            44:cd:8c:e6:a4:60:ff:3f:09:af:02:e7:68:5e:f2:0f:e6:a0:39:fe
    Signature Algorithm: sha256WithRSAEncryption
        Issuer: C=CN, ST=ShenZhen, L=ShenZhen, O=k8s, OU=System, CN=kubernetes
        Validity
            Not Before: Dec 13 06:35:00 2019 GMT
            Not After : Nov 19 06:35:00 2119 GMT
        Subject: C=CN, ST=ShenZhen, L=ShenZhen, O=k8s, OU=System, CN=kubernetes
        Subject Public Key Info:
            Public Key Algorithm: rsaEncryption
                Public-Key: (2048 bit)
                Modulus:
                    00:d7:91:4f:90:56:fb:ab:a9:de:c4:98:9e:d7:e6:
                    45:db:5a:14:9a:76:78:6a:4c:db:3c:47:3c:e7:1c:
                    3c:37:4e:8a:cf:9c:a1:8a:4c:51:4c:cd:45:b0:03:
                    87:06:b9:20:2c:3a:51:f9:21:55:1c:90:7c:f8:93:
                    bc:6a:48:05:3d:8b:74:fd:f2:f1:e6:5e:ad:b4:a8:
                    f6:6d:f9:63:9e:e4:b4:cc:68:9e:90:d7:ef:de:ce:
                    c1:1d:1b:68:59:68:5e:5f:7d:5c:f3:49:4f:18:72:
                    be:b5:c8:af:e2:8d:34:9c:d2:68:b7:8c:26:69:cc:
                    a5:f4:ca:69:2d:d7:21:f5:19:2e:b2:b5:97:16:87:
                    9f:9c:fd:01:97:c2:0e:20:b4:88:27:9a:37:9a:af:
                    0a:cf:82:4f:26:24:cb:07:ac:8c:b1:34:20:42:22:
                    00:b2:b0:98:c5:53:01:fb:32:aa:15:1b:7e:39:44:
                    ae:af:6e:c3:65:96:f6:38:7a:87:37:d0:31:63:d8:
                    a4:15:13:f2:56:da:e6:09:45:2b:46:2c:cb:63:db:
                    f7:ba:7f:44:0a:36:39:7c:cc:5b:42:e5:56:c7:7f:
                    dd:64:5c:f2:4a:af:d3:a9:d1:6e:06:27:57:09:4d:
                    db:08:62:87:66:c8:2c:36:00:41:f1:90:f6:5f:68:
                    20:3d
                Exponent: 65537 (0x10001)
        X509v3 extensions:
            X509v3 Key Usage: critical
                Digital Signature, Key Encipherment
            X509v3 Extended Key Usage:
                TLS Web Server Authentication, TLS Web Client Authentication
            X509v3 Basic Constraints: critical
                CA:FALSE
            X509v3 Subject Key Identifier:
                3D:3F:FA:B8:36:D7:FE:B1:59:BE:B1:F5:C1:5D:88:3D:BC:78:9F:87
            X509v3 Authority Key Identifier:
                keyid:40:A2:D4:30:22:12:2E:C2:FB:A2:55:2C:CB:F0:F6:3E:4D:B8:02:03

            X509v3 Subject Alternative Name:
                DNS:kubernetes, DNS:kubernetes.default, DNS:kubernetes.default.svc, DNS:kubernetes.default.svc.cluster, DNS:kubernetes.default.svc.cluster.local, IP Address:127.0.0.1, IP Address:172.20.0.112, IP Address:172.20.0.113, IP Address:172.20.0.114, IP Address:172.20.0.115, IP Address:10.254.0.1
    Signature Algorithm: sha256WithRSAEncryption
         63:50:f6:2a:03:c7:35:dd:e9:10:8d:2f:b3:27:9a:64:f3:e1:
         11:8a:18:1e:fa:6d:85:30:11:b4:59:a3:6c:86:cd:2b:5c:67:
         17:4f:aa:0d:bb:4c:ee:c8:af:e7:3d:61:6d:03:9d:14:6f:00:
         48:56:59:b5:76:13:a9:30:23:e0:b2:d2:12:64:0c:60:0d:76:
         ec:c6:4f:b1:bc:24:01:7a:48:c6:fd:9e:5d:68:da:b9:a1:ad:
         30:7a:ba:90:e2:e3:4e:b4:92:1b:c5:f2:8c:c1:b0:3d:fc:14:
         d2:46:e8:f8:22:8f:d9:4d:85:4f:58:6b:0f:84:78:06:b4:b9:
         92:b9:0d:bd:1d:95:e9:0d:42:d3:fd:dd:2a:59:60:3f:63:35:
         eb:07:25:d2:ea:0d:19:a6:f3:dc:92:8e:ee:73:04:15:5e:97:
         e8:da:51:c3:69:49:96:36:c7:cc:5b:e5:e5:cb:e5:ce:9f:21:
         6f:6b:56:16:bf:85:ad:1c:8c:91:c1:91:0a:90:18:e2:4a:b0:
         32:58:33:ef:55:8e:8f:4a:e3:0f:b8:f7:41:04:65:89:e1:1b:
         d8:41:28:6e:84:c3:1c:8e:a9:a0:8a:42:e4:fe:d7:fe:0e:24:
         dc:74:37:fa:5e:be:20:69:c5:9a:5a:e6:83:1c:0b:9e:e1:43:
         ef:4f:7a:37

mkdir -p /etc/kubernetes/ssl
cp *.pem /etc/kubernetes/ssl

apiVersion: apps/v1beta2 # for versions before 1.8.0 use apps/v1beta1
kind: Deployment
metadata:
  name: nginx-deployment
spec:
  replicas: 3
  selector:
    matchLabels:
      app: nginx
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
      - name: nginx
        image: nginx:1.7.9
        ports:
        - containerPort: 80

#create command
kubectl create -f https://k8s.io/docs/user-guide/nginx-deployment.yaml --record
#output
deployment "nginx-deployment" created

kubectl delete pod <pod_name> --namespace=<namespace>  --force --grace-period=0

apiVersion: v1
kind: Pod
metadata:
  name: string
  namaspace: string
  labels:
  - name: string
      annotations:
  - name: string
  spec:
    containers:
  - name: string
    images: string
    imagePullPolice: [Always | Never | IfNotPresent]
    command: [string]
    args: [string]
    workingDir: string
    volumeMounts:
    - name: string
      mountPath: string
      readOnly: boolean
      ports:
    - name: string
      containerPort: int
      hostPort: int
      protocol: string
      env:
    - name: string
      value: string
      resources:
      limits:
        cpu: string
        memory: string
      requests:
        cpu: string
        memory: string
      livenessProbe:
      exec:
        command: [string]
      httpGet:
        path: string
        port: int
        host: string
        scheme: string
        httpHeaders:
        - name: string
          value: string
          tcpSocket:
            port: int
          initialDelaySeconds: number
          timeoutSeconds: number
          periodSeconds: number
          successThreshold: 0
          failureThreshold: 0
          securityContext:
          privileged: false
          restartPolicy: [Always | Never | OnFailure]   
          nodeSelector: object
          imagePullSecrets:
  - name: string
      hostNetwork: false
        volumes:
  - name: string
    emptyDir: {}
    hostPath:
      path: string
    secret:
      secretName: string
      items:
      - key: string
        path: string
        configMap:
          name: string
          items:
      - key: string
        path: string

apiVersion: v1
kind: Pod
metadata:
  name: static-web
  labels:
    name: static-web
spec:
  containers:
  - name: static-web
  image: nginx
  ports:
  - name: web
    containerPort: 80